//
//  WebExtractorController.m
//  WebClipper
//
//  Created by Daniel Braun on 05/02/08.
//  Copyright 2008 __MyCompanyName__. All rights reserved.
//

#import "WebExtractorController.h"
#import "WCWebLoader.h"
#import "WCExtractSetArray.h"
#import "WCWebExtractSet.h"
#import "WCUrlAccess.h"



@implementation WebExtractorController


#pragma mark -
#pragma mark *** init, dealloc... ***


- (void) dealloc
{
	[delegate release];
	[loader release];
	[extracted release];
	[_aextracted release];
	[esets release];
	[delegate release];
	[super dealloc];
}

- (id) init
{
	self = [super init];
	if (self != nil) {
	}
	return self;
}



#pragma mark -
#pragma mark *** load and patch ***


/* step 1 : load an .webExtract file, probably from application resources */
- (BOOL) loadFile:(NSString *)p fromBundleResources:(BOOL)fr
{
	//NSLog(@"laod esets %@\n", p ? p : @"embedded");
	if (!p) {
		p=@"main.webExtract";
	}	
	if (fr) p=[NSString stringWithFormat:@"%@/%@", [[NSBundle mainBundle] resourcePath
					], p];
	NSLog(@"opening %@\n", p);
	if (!p) {
		NSLog(@"bad, webExtract %@ not found\n",p);
		return NO;
	}
	NSError *err=nil;
	NSData *data=[NSData dataWithContentsOfFile:p 
				    options:0
				      error:&err];
	if (!data) {
		char tmp[1024*2];
		NSLog(@"error reading file %@ : %@\n", p, err);
		NSLog(@"curdir %s\n", getcwd(tmp, sizeof(tmp)));
		return NO;
	}
	NSDictionary *info = [NSKeyedUnarchiver unarchiveObjectWithData:data];
	if (!info) {
		NSLog(@"unarchive failed\n");
		goto badfile;
	}
	
	
	
	[esets release]; esets=nil;
	
	id esetsa=[info objectForKey:@"esets"];
	if ([esetsa isKindOfClass:[NSNull class]]) {
		return NO;
	} else if ([esetsa isKindOfClass:[WCExtractSetArray class]]) {
		esets=[esetsa retain];
	} else if ([esetsa isKindOfClass:[NSArray class]]) {
		// compatibility with pre-20070407 files
		esets=[[WCExtractSetArray alloc]initWithArray:esetsa];
	} else {
		NSAssert(0, @"bad class for esets");
	}
	
	if (!esets) return NO;
	if ([esets count]<1) goto badfile;
	//extracted=[[esets parsedOtherVariables]retain];
	//[self parseVariables:extracted];	
	return YES;
badfile:
	return NO;
}



/* step 2 : possibly - for complex case - change pages url or user input values */
- (NSURL *) urlForPage:(NSString *)page
{
	WCWebExtractSet *s=[esets esetForPage:page];
	NSURL *url=[[s urlaccess]url];
	return url;
}
- (void) setUrl:(NSURL *)url forPage:(NSString *)page
{
	WCWebExtractSet *s=[esets esetForPage:page];
	[[s urlaccess]setUrl:url];
}

- (NSDictionary *) userInputsInPage:(NSString *)page
{
	WCWebExtractSet *s=[esets esetForPage:page];
	NSDictionary *d=[[s urlaccess]variables];
	return d;
}
- (void) setUserInput:(NSString *)varname inPage:(NSString *)page toValue:(NSString *)value
{
	WCWebExtractSet *s=[esets esetForPage:page];
	NSDictionary *d=[[s urlaccess]variables];
	if (!d) return;
	if (!value) return;
	if ([d objectForKey:varname]) {
		NSMutableDictionary *dm=[d mutableCopy];
		NSString *ve=[value stringByAddingPercentEscapesUsingEncoding:NSASCIIStringEncoding];
		if (!ve) ve=[value stringByAddingPercentEscapesUsingEncoding:NSISOLatin1StringEncoding];
		if (!ve) ve=[value stringByAddingPercentEscapesUsingEncoding:NSUTF8StringEncoding];
		[dm setObject:ve  forKey:varname];
		[[s urlaccess]setVariables:dm];
	}
}


#pragma mark -
#pragma mark *** fetch data ***


/* step 3 : fetch pages */

- (void) pageLoadedBy:(NSObject <WCWebLoadProtocol> *)ctrler private:(WCWebExtractSet*)wes
{
	// page reuslt update
#if 0
	NSData *h=[ctrler dhtml];
	
	NSAttributedString *as=[[[NSAttributedString alloc] initWithHTML:h
						      documentAttributes:nil]autorelease];
	[self _addPage:as];
#endif
	
	NSDictionary *exv=[wes processExtractionWithController:ctrler];
	if (![exv count]) goto extraction_done; // no new variables
	
	[self willChangeValueForKey:@"extractedArray"];
	[self willChangeValueForKey:@"extractedDictionary"];
	[_aextracted release]; _aextracted=nil;
	
	NSMutableDictionary *md=[NSMutableDictionary dictionaryWithCapacity:[exv count]];
	NSEnumerator *e=[exv keyEnumerator];
	NSString *k;
	while (k=[e nextObject]) {
		id v;
		NSString *nk;
		v=[exv objectForKey:k];
		nk=[NSString stringWithFormat:@"%@.%@", [wes name], k];
		[md setObject:v forKey:nk];
	}
	if (!extracted) extracted=[md retain];
	else [extracted addEntriesFromDictionary:md];
	
	[self didChangeValueForKey:@"extractedArray"];
	[self didChangeValueForKey:@"extractedDictionary"];
	
	NSDictionary *bodyv;
extraction_done:
	bodyv=[wes bodyVariables];
	wes=[wes nextExtractSet];
	if ([delegate respondsToSelector:@selector(dataFetched)]) {
		[delegate dataFetched];
	}
	if (wes) {
		NSString *content=nil;
		NSURL *u=[wes actualUrlOrContent:&content];
		if (!u) {
		} else {
			[loader setUrl:u delegateQuery:nil urlAccess:[wes urlaccess] bodyVariables:bodyv];
			if (content) {
				[loader loadHtml: content andPerform:@selector(pageLoadedBy:private:) 
					  target:self private:wes];
			} else {
				[loader loadPageAndPerform:@selector(pageLoadedBy:private:) 
						    target:self private:wes];
			}
		}
	} else {
		[self didEndLoad];

		if ([delegate respondsToSelector:@selector(allDataFetched)]) {
			[delegate allDataFetched];
		}
	}
}


- (void) fetchDataWithDelegate:(id)aDelegate
{
	[delegate release];
	delegate=[aDelegate retain];
	
	if (![self willStartLoad]) {
		//NSLog(@"too many queries, waiting.. (%@)\n", self);
		[self performSelector:@selector(fetchDataWithDelegate:) withObject:aDelegate 
			   afterDelay:1+(rand()%5)];
		return;
	}
	[loader release];
	loader=[[WCWebLoader alloc]init];
	
	WCWebExtractSet *wes=[esets objectAtIndex:0];
	[loader setUrl:[wes actualUrl] delegateQuery:nil urlAccess:[wes urlaccess] bodyVariables:nil];
	[loader loadPageAndPerform:@selector(pageLoadedBy:private:) 
			    target:self private:wes];	
}

/* step 4: get values */
- (NSDictionary *) extractedDictionary
{
	return extracted;
}
- (NSArray *) extractedArray
{
	return _aextracted;
}



#pragma mark -
#pragma mark *** copy duplicate ***

- (id) copy
{
	WebExtractorController *c=[[WebExtractorController alloc]init];
	c->esets=[esets copy];
	return c;
}

#pragma mark -
#pragma mark *** request limitations ***

//static int numReq=0;
//static int maxReq=1;
// bucket
static NSDate *bD=nil;
static float bLevel=0;
static float bR=0.2;
static float bMax=5;

- (BOOL) willStartLoad
{
	if (bD) {
		NSTimeInterval di=-[bD timeIntervalSinceNow];
		bLevel=bLevel-bR*di;
		[bD release];
		bD=[[NSDate date] retain];
		if (bLevel<0) bLevel=0;
		//NSLog(@"blevel %g\n", bLevel);
		if (bLevel>bMax) return NO;
		bLevel+=1.0;
		return YES;
	}
	bD=[[NSDate date] retain];
	bLevel=1.0;
	return YES;
#if 0
	if (numReq>=maxReq) return NO;
	numReq++;
	return YES;
#endif
}

- (void) didEndLoad
{
#if 0
	numReq--;
	NSAssert(numReq>=0, @"neg req");
#endif
}

- (NSString *)html
{
	return [loader html];
}
- (NSData *)dhtml
{
	return [loader dhtml];
}
@end
